{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total label: 6391 \n"
     ]
    }
   ],
   "source": [
    "train_data_dir = '../Train_label.csv'\n",
    "tags = []\n",
    "train_df = pd.read_csv(train_data_dir)\n",
    "train_df['type'] = train_df['type'].astype(str)\n",
    "for i in range(train_df['type'].shape[0]):\n",
    "    for tag in train_df['type'].iloc[i].split(','):\n",
    "        tags.append(tag)\n",
    "print('total label: %d ' % len(tags))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'7', '1', '9', '6', '4', '3', '8', '5', '2'}\n",
      "9\n"
     ]
    }
   ],
   "source": [
    "label=set(tags)\n",
    "print(label)\n",
    "print(len(label))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'1': 373,\n",
       " '2': 1296,\n",
       " '3': 815,\n",
       " '4': 499,\n",
       " '5': 841,\n",
       " '6': 594,\n",
       " '7': 471,\n",
       " '8': 896,\n",
       " '9': 606}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_num = {}\n",
    "for key in tags:\n",
    "    label_num[key] = label_num.get(key, 0) + 1\n",
    "label_num"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total img: 6391 \n"
     ]
    }
   ],
   "source": [
    "for i in range(train_df['type'].shape[0]):\n",
    "    train_df['FileName'].iloc[i] = train_df['FileName'].iloc[i].split('/')[-1]\n",
    "train_img_paths = list(train_df['FileName'])\n",
    "local_path = '../Train/'\n",
    "for i in range(len(train_img_paths)):\n",
    "    train_img_paths[i] = local_path + train_img_paths[i]\n",
    "print('total img: %d ' % (len(train_img_paths)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total img: 4260 \n"
     ]
    }
   ],
   "source": [
    "test_data_dir = '../submit_example.csv'\n",
    "test_img_paths = []\n",
    "test_df = pd.read_csv(test_data_dir)\n",
    "for i in range(test_df['FileName'].shape[0]):\n",
    "    test_img_paths.append(test_df['FileName'].iloc[i].split('/')[-1])\n",
    "local_path = '../Train/'\n",
    "for i in range(len(test_img_paths)):\n",
    "    test_img_paths[i] = local_path + test_img_paths[i]\n",
    "print('total img: %d ' % (len(test_img_paths)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10651"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "img_paths = []\n",
    "img_paths = train_img_paths\n",
    "img_paths.extend(test_img_paths)\n",
    "len(img_paths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "ename": "IndexError",
     "evalue": "invalid index to scalar variable.",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-9-3369d6a78d60>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     25\u001b[0m     \u001b[0mimg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mimg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat32\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;36m255.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m         \u001b[0mmeans\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mimg\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     28\u001b[0m         \u001b[0mstdevs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mimg\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m \u001b[0;31m# print(num_imgs)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mIndexError\u001b[0m: invalid index to scalar variable."
     ],
     "output_type": "error"
    }
   ],
   "source": [
    "import os\n",
    "import cv2\n",
    "import numpy as np\n",
    "lines = img_paths\n",
    "path = 'mean_std.txt'\n",
    "means = [0, 0, 0]\n",
    "stdevs = [0, 0, 0]\n",
    "\n",
    "index = 1\n",
    "num_imgs = 0\n",
    "# with open(path, 'r') as f:\n",
    "#     lines = f.readlines()\n",
    "#     # random.shuffle(lines)\n",
    "#     print(lines)\n",
    "for line in lines:\n",
    "#     print(line)\n",
    "#     print('{}/{}'.format(index, len(lines)))\n",
    "    index += 1\n",
    "    a = os.path.join(line)\n",
    "    # print(a[:-1])\n",
    "    num_imgs += 1\n",
    "    img = cv2.imread(a)\n",
    "    img = np.asarray(img)\n",
    "#     print(img)\n",
    "    img = img.astype(np.float32) / 255.\n",
    "    for i in range(3):\n",
    "        means[i] += img[:, :, i].mean()\n",
    "        stdevs[i] += img[:, :, i].std()\n",
    "# print(num_imgs)\n",
    "means.reverse()\n",
    "stdevs.reverse()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "means = np.asarray(means) / num_imgs\n",
    "stdevs = np.asarray(stdevs) / num_imgs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "normMean = [0.50435057 0.52046513 0.50014097]\n",
      "normStd = [0.21178572 0.20667593 0.20981381]\n",
      "transforms.Normalize(normMean = [0.50435057 0.52046513 0.50014097], normStd = [0.21178572 0.20667593 0.20981381])\n"
     ]
    }
   ],
   "source": [
    "print(\"normMean = {}\".format(means))\n",
    "print(\"normStd = {}\".format(stdevs))\n",
    "print('transforms.Normalize(normMean = {}, normStd = {})'.format(means, stdevs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.utils import class_weight\n",
    "class_weight = class_weight.compute_class_weight('balanced',np.unique(tags),tags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class_weight_dict = dict(enumerate(class_weight))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: 1.9037831397080727,\n",
       " 1: 0.5479252400548696,\n",
       " 2: 0.8713019768234492,\n",
       " 3: 1.4230683589401025,\n",
       " 4: 0.8443651737349716,\n",
       " 5: 1.1954732510288066,\n",
       " 6: 1.5076669025713612,\n",
       " 7: 0.7925347222222222,\n",
       " 8: 1.1718005133846718}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_weight_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1.0,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1.0,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
